package com.flute.icrawler.htmlparser.util;

import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;

import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import com.flute.icrawler.htmlparser.HtmlParserException;
import com.flute.icrawler.htmlparser.ParserFilter;

public class HtmlParserUtil {

	public HtmlParserUtil() {
	}

	public static String[] getHTML(Document document, ParserFilter filter) {
		try {
			List<String> list = new ArrayList<String>();
			NodeList nodes = (NodeList) xpath.evaluate(filter.xpath, document,
					XPathConstants.NODESET);
			int len = nodes.getLength();
			for (int i = 0; i < len; i++) {
				Node node = nodes.item(i);
				StringWriter writer = new StringWriter();
				TransformerFactory
						.newInstance()
						.newTransformer()
						.transform(new DOMSource(node),
								new StreamResult(writer));
				list.add(writer
						.toString()
						.replaceAll(
								"\\<\\?xml version=\\\"1.0\\\"\\s*encoding=\\\".*?\\\"\\?\\>",
								"").replaceAll("(\r\n|\n\r){2,}", "\r\n"));
			}

			return (String[]) list.toArray(new String[0]);
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
	}

	public static String[] getContent(Document document, ParserFilter filter) {
		try {
			List<String> list = new ArrayList<String>();
			NodeList nodes = (NodeList) xpath.evaluate(filter.xpath, document,
					XPathConstants.NODESET);
			Pattern pattern = null;
			boolean doPattern = false;
			if (filter != null && filter.pattern != null
					&& !filter.pattern.trim().equals("")) {
				pattern = Pattern.compile(filter.pattern.trim());
				doPattern = true;
			}
			int len = nodes.getLength();
			for (int i = 0; i < len; i++) {
				Node node = nodes.item(i);
				StringBuffer buffer = new StringBuffer("");
				getNodeValue(node, buffer);
				if (!buffer.toString().trim().equals(""))
					if (doPattern) {
						for (Matcher matcher = pattern.matcher(buffer
								.toString()); matcher.find(); list.add(matcher
								.group()))
							;
					} else {
						list.add(buffer.toString());
					}
			}

			return (String[]) list.toArray(new String[0]);
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
	}

	public static String[] getImgs(Document document, ParserFilter filter)
			throws HtmlParserException {
		try {
			List<String> list = new ArrayList<String>();
			NodeList nodes = (NodeList) xpath.evaluate("//IMG", document,
					XPathConstants.NODESET);
			Pattern pattern = null;
			boolean doPattern = false;
			if (filter != null && filter.pattern != null
					&& !filter.pattern.trim().equals("")) {
				pattern = Pattern.compile(filter.pattern.trim());
				doPattern = true;
			}
			int len = nodes.getLength();
			for (int i = 0; i < len; i++) {
				Element element = (Element) nodes.item(i);
				String src = element.getAttribute("src");
				if (src != null && !src.trim().equals(""))
					if (doPattern) {
						Matcher matcher = pattern.matcher(src);
						if (matcher.find())
							list.add(src);
					} else {
						list.add(src);
					}
			}

			return (String[]) list.toArray(new String[0]);
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
	}

	public static String[] getLinks(Document document, ParserFilter filter,
			String source) throws HtmlParserException {
		try {
			HashSet<String> set = new HashSet<String>();
			NodeList nodes = (NodeList) xpath.evaluate("//A|//IFRAME|//FRAME",
					document, XPathConstants.NODESET);
			Pattern pattern = null;
			boolean doPattern = false;
			if (filter != null && filter.pattern != null
					&& !filter.pattern.trim().equals("")) {
				pattern = Pattern.compile(filter.pattern.trim());
				doPattern = true;
			}
			int len = nodes.getLength();
			for (int i = 0; i < len; i++) {
				String uri = "";
				Element element = (Element) nodes.item(i);
				String tag = element.getNodeName();
				if (tag.equals("A"))
					uri = element.getAttribute("href");
				else
					uri = element.getAttribute("src");
				if (uri != null && !uri.trim().equals("") && !uri.equals("#"))
					if (doPattern) {
						Matcher matcher = pattern.matcher(uri);
						if (matcher.find())
							set.add(fillUrl(source, uri));
					} else {
						set.add(fillUrl(source, uri));
					}
			}

			return (String[]) (new ArrayList(set)).toArray(new String[0]);
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
	}

	public String[] getNodesAttribute(Document document, ParserFilter filter,
			String attribute) throws HtmlParserException {
		try {
			HashSet<String> set = new HashSet<String>();
			NodeList nodes = (NodeList) xpath.evaluate(filter.xpath, document,
					XPathConstants.NODESET);
			Pattern pattern = null;
			boolean doPattern = false;
			if (filter != null && filter.pattern != null
					&& !filter.pattern.trim().equals("")) {
				pattern = Pattern.compile(filter.pattern.trim());
				doPattern = true;
			}
			int len = nodes.getLength();
			for (int i = 0; i < len; i++) {
				Element element = (Element) nodes.item(i);
				String atrr = element.getAttribute(attribute);
				if (atrr != null && !atrr.trim().equals(""))
					if (doPattern) {
						Matcher matcher = pattern.matcher(atrr);
						if (matcher.find())
							set.add(atrr);
					} else {
						set.add(atrr);
					}
			}

			return (String[]) (new ArrayList(set)).toArray(new String[0]);
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
	}

	private static void getNodeValue(Node node, StringBuffer buffer) {
		if (node.getNodeType() == 3)
			buffer.append(node.getNodeValue());
		else if (node.getNodeType() == 1) {
			Element element = (Element) node;
			String tag = element.getTagName().toLowerCase();
			if (!tag.equals("script") && !tag.equals("style")
					&& !tag.equals("link") && !tag.equals("iframe")
					&& !tag.equals("frame") && node.hasChildNodes()) {
				NodeList nodes = node.getChildNodes();
				for (int i = 0; i < nodes.getLength(); i++) {
					Node child = nodes.item(i);
					getNodeValue(child, buffer);
				}

			}
		} else if (node.getNodeType() == 2) {
			String text = node.getNodeValue();
			if (text != null)
				buffer.append(text);
		}
	}

	private static void transform(Node node, StringBuffer buffer) {
		if (node.getNodeType() == 3) {
			String text = node.getNodeValue();
			if (text != null && !text.equals(""))
				buffer.append(text);
		} else if (node.getNodeType() == 1) {
			String tag = node.getNodeName().toUpperCase();
			buffer.append("<").append(tag);
			NamedNodeMap attrs = node.getAttributes();
			for (int i = 0; i < attrs.getLength(); i++) {
				Node attr = attrs.item(i);
				buffer.append(" ").append(attr.getNodeName().toLowerCase())
						.append("='").append(attr.getNodeValue()).append("'");
			}

			buffer.append(">");
			if (node.hasChildNodes()) {
				NodeList nodes = node.getChildNodes();
				for (int j = 0; j < nodes.getLength(); j++)
					transform(nodes.item(j), buffer);

			}
			buffer.append("</").append(tag).append(">");
		} else if (node.getNodeType() == 9)
			transform(((Node) (((Document) node).getDocumentElement())), buffer);
		else if (node.getNodeType() == 2)
			buffer.append(node.getNodeValue());
	}

	public static String getHTML(Document document) {
		StringBuffer buffer = new StringBuffer();
		transform(document, buffer);
		return buffer.toString().replaceAll("(\r\n|\n\r){2,}", "\r\n");
	}

	private static String fillUrl(String source, String uri) {
		if (uri.startsWith("http"))
			return uri;
		if (uri.startsWith("www."))
			return (new StringBuilder("http://")).append(uri).toString();
		if (uri.startsWith("/")) {
			Pattern p = Pattern.compile("http://(\\w+\\.)*\\w+(?=/)");
			Matcher m = p.matcher(source);
			if (m.find())
				return (new StringBuilder(String.valueOf(m.group()))).append(
						uri).toString();
			else
				return uri;
		} else {
			return (new StringBuilder(String.valueOf(source.substring(0,
					source.lastIndexOf("/") + 1)))).append(uri).toString();
		}
	}

	private static XPath xpath = XPathFactory.newInstance().newXPath();

}
